
import requests
from requests.exceptions import RequestException
import re
from openpyxl import workbook 
from openpyxl import load_workbook  
import time


def get_one_page(url):
    try:
        response = requests.get(url)
        if response.status_code == 200: 
            #print("get ok")
            return  response.text
        print(response.status_code)
        return None
    except RequestException:
        print("get err")
        return None


def parse_one_total_page(html):
    pattern = re.compile('<a class="next" href(.*?)</a>',re.S)
    items = re.findall(pattern,html)
    return items


def parse_one_index_page(html):

    pattern = re.compile('<a class="goodImg" href="(.*?)">',re.S)

    items = re.findall(pattern,html)

    return items

def parse_one_goods_page(htmlGoods,goodsUrl):
    global ws  

    patternGoods = re.compile('<div class="goodsTitle">\n\t\t\t\t\t\t(.*?)\t.*?<font class="brief">.*?<div class="goodsMes" data-stock=".*?" data-price="(.*?)" data-promote=".*?" data-vip=".*?">.*?<div class="name">货号</div>.*?<div class="content2">(.*?)</div>',re.S)
    itemsGoods = re.findall(patternGoods,htmlGoods)
    
    for item in itemsGoods:

        ws.append([item[2],item[0],item[1],goodsUrl]) 
        

def main(page):
    pageUrl = 'http://www.dfrobot.com.cn/category.php?id=216&price_min=0&price_max=0&page='+str(page)+'&sort=goods_id&order=DESC'
        
    html = get_one_page(pageUrl)
    print('main running page '+str(page))
    for item in parse_one_index_page(html):
        #print(item)
        goodsUrl = 'http://www.dfrobot.com.cn/'+item
        goodsHtml = get_one_page(goodsUrl) 
        parse_one_goods_page(goodsHtml,goodsUrl) 
  
if __name__ == '__main__':
    running = True

    wb = workbook.Workbook() 
    ws = wb.active  

    ws.append(['SKU', 'name', 'price','link'])

    page = 1
    print('calculation page total...')
    while running:
        pageUrl = 'http://www.dfrobot.com.cn/category.php?id=216&price_min=0&price_max=0&page='+str(page)+'&sort=goods_id&order=DESC'
        if parse_one_total_page(get_one_page(pageUrl)) ==[]: 
            running = False
        main(page) 
        page += 1
        '''if page==2:
            running=0'''

    print('page total is : '+str(page-1))
    print('save...')
    
    print(time.strftime('%Y%m%d%H%M%S',time.localtime(time.time())))
    
    wb.save('result_'+time.strftime('%Y%m%d%H%M%S',time.localtime(time.time()))+'.xlsx')

    print('save done')



